import fs from 'fs'
import * as cheerio from 'cheerio'
import axios from 'axios'
import path from 'path'
import { fileURLToPath } from 'url'
import process from 'node:process'

const dirname = path.dirname(fileURLToPath(import.meta.url))

const savePath = path.join(dirname, './src/data.json') // 数据保存位置
const startURLs = [
  {
    url: 'https://hanyu.baidu.com/s?wd=%E4%BA%95%E4%BA%95%E6%9C%89%E6%9D%A1&device=pc&from=home',
    get($) {
      if ($('#idiom-body').length) return [$('#term-header h2 strong').text()]
      return []
    }
  },
  {
    url: 'https://chengyu.qianp.com/',
    get($, url) {
      if (url.pathname.startsWith('/cy/')) return [$('h1').text()]
      return []
    }
  }
]
const threadsNum = 20
const delay = 0
const configPath = path.join(dirname, './crawlingData-config.json')

try {
  fs.accessSync(savePath, fs.constants.W_OK | fs.constants.R_OK)
} catch {
  fs.writeFileSync(savePath, '[]', { encoding: 'utf-8' })
}
let config = {
  visited: [],
  link: []
}
try {
  config = JSON.parse(fs.readFileSync(configPath, { encoding: 'utf-8' }))
} catch {
  //
}
let data = JSON.parse(fs.readFileSync(savePath, { encoding: 'utf-8' }))

config.link.push(
  ...startURLs
    .map((e) => e.url)
    .filter((e) => !config.link.includes(e) && !config.visited.includes(e))
)
let visited = config.visited.map((e) => new URL(e))
const link = config.link.map((e) => new URL(e))
const working = []
let errorLink = []

setTimeout(function fn() {
  while (working.length < threadsNum)
    if (link.length) {
      const url = link.pop()
      const t = {
        url,
        promise: axios.get(url.href, {
          responseType: 'arraybuffer',
          headers: {
            'User-Agent':
              'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.41'
          }
        })
      }
      working.push(t)
      t.promise
        .then((e) => {
          const $ = cheerio.load(e.data, { baseURI: url })
          startURLs
            .filter((e) => new URL(e.url).hostname === url.hostname)
            .forEach((e) => {
              data.push(...e.get($, url))
            })
          $('a[href]').each((_, el) => {
            try {
              const t1 = $(el).prop('href')
              if (t1) {
                const url1 = new URL(t1, url)
                if (
                  startURLs.some((e) => url1.hostname === new URL(e.url).hostname) &&
                  !working.some((e) => e.url.href === url1.href) &&
                  !link.some((e) => e.href === url1.href) &&
                  !visited.some((e) => e.href === url1.href)
                )
                  link.push(url1)
              }
            } catch {
              //
            }
          })
          const i = working.findIndex((e) => e.url.href === url.href)
          if (i !== -1) {
            visited.push(url)
            working.splice(i, 1)
          }
          errorLink = errorLink.filter((e) => e.href !== url.href)
        })
        .catch((e) => {
          console.error('访问出错', url.href, e.message)
          const i = working.findIndex((e) => e.url.href === url.href)
          if (i !== -1) {
            if (!errorLink.some((e) => e.href === url.href)) errorLink.push(url)
            working.splice(i, 1)
            setTimeout(() => {
              if (
                !working.some((e) => e.url.href === url.href) &&
                !link.some((e) => e.href === url.href) &&
                !visited.some((e) => e.href === url.href)
              )
                link.push(url)
            }, 60000)
          }
        })
    } else if (!working.length) {
      visited = []
      link.push(...startURLs.map((e) => new URL(e.url)))
      setTimeout(fn, 60000)
    } else break
  setTimeout(fn, delay)
}, delay)

let len = data.length
setTimeout(function fn() {
  data = [...new Set(data)]
  console.log(
    '数据长度：',
    data.length,
    ' | 待爬取链接：',
    link.length,
    ' | 工作中链接：',
    working.length,
    ' | 已爬取链接：',
    visited.length,
    ' | 异常链接：',
    errorLink.length
  )
  if (data.length !== len) {
    len = data.length
    fs.writeFileSync(savePath, JSON.stringify(data), {
      encoding: 'utf-8'
    })
  }
  setTimeout(fn, 10000)
}, 10000)

process.on('uncaughtException', (...a) => {
  console.error('uncaughtException', a)
  process.exit()
})
process.on('SIGINT', () => process.exit())
process.on('SIGHUP', () => process.exit())
process.on('exit', () => {
  data = [...new Set(data)]
  console.log(
    '数据长度：',
    data.length,
    ' | 待爬取链接：',
    link.length,
    ' | 工作中链接：',
    working.length,
    ' | 已爬取链接：',
    visited.length,
    ' | 异常链接：',
    errorLink.length
  )
  fs.writeFileSync(savePath, JSON.stringify(data), {
    encoding: 'utf-8'
  })
  fs.writeFileSync(
    configPath,
    JSON.stringify({ visited, link: [...link, ...working.map((e) => e.url), ...errorLink] }),
    { encoding: 'utf-8', flag: 'w' }
  )
  console.log('exit')
})
